{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-12-28T07:52:31.531700Z",
     "start_time": "2018-12-28T07:52:23.447638Z"
    },
    "code_folding": [],
    "collapsed": true,
    "run_control": {
     "marked": false
    }
   },
   "outputs": [],
   "source": [
    "from tensorflow.python.layers import core as core_layers\n",
    "import tensorflow as tf\n",
    "import numpy as np\n",
    "import time\n",
    "import os\n",
    "import Util.myResidualCell\n",
    "from Util.bleu import BLEU\n",
    "from tensorflow.python.ops import array_ops\n",
    "import random\n",
    "import pickle as cPickle\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "\n",
    "class VRAE:\n",
    "    def __init__(self, dp, rnn_size, n_layers, latent_dim, var, encoder_embedding_dim, decoder_embedding_dim, max_infer_length,\n",
    "                 sess=tf.Session(), lr=0.001, grad_clip=5.0, is_jieba=False, beam_width=5, force_teaching_ratio=1.0, beam_penalty=0.0,\n",
    "                residual=False, output_keep_prob=0.5, input_keep_prob=0.9, cell_type='lstm', reverse=False, \n",
    "                latent_weight=0.1, beta_decay_period=10, beta_decay_offset=5, decay_scheme='luong234', is_save=True):\n",
    "        \n",
    "        self.rnn_size = rnn_size\n",
    "        self.latent_dim = latent_dim\n",
    "        self.n_layers = n_layers\n",
    "        self.grad_clip = grad_clip\n",
    "        self.is_jieba = is_jieba\n",
    "        self.var = var\n",
    "        self.dp = dp\n",
    "        self.step = 0\n",
    "        self.encoder_embedding_dim = encoder_embedding_dim\n",
    "        self.decoder_embedding_dim = decoder_embedding_dim\n",
    "        self.beam_width = beam_width\n",
    "        self.latent_weight = latent_weight\n",
    "        self.beam_penalty = beam_penalty\n",
    "        self.max_infer_length = max_infer_length\n",
    "        self.residual = residual\n",
    "        self.is_save = is_save\n",
    "        self.decay_scheme = decay_scheme\n",
    "        if self.residual:\n",
    "            assert encoder_embedding_dim == rnn_size\n",
    "            assert decoder_embedding_dim == rnn_size\n",
    "        self.reverse = reverse\n",
    "        self.cell_type = cell_type\n",
    "        self.force_teaching_ratio = force_teaching_ratio\n",
    "        self._output_keep_prob = output_keep_prob\n",
    "        self._input_keep_prob = input_keep_prob\n",
    "        self.beta_decay_period = beta_decay_period\n",
    "        self.beta_decay_offset = beta_decay_offset\n",
    "        self.sess = sess\n",
    "        self.lr=lr\n",
    "        self.build_graph()\n",
    "        self.sess.run(tf.global_variables_initializer())\n",
    "        self.saver = tf.train.Saver(tf.global_variables(), max_to_keep = 30)\n",
    "        self.summary_placeholders, self.update_ops, self.summary_op = self.setup_summary()\n",
    "        \n",
    "    # end constructor\n",
    "\n",
    "    def build_graph(self):\n",
    "        self.register_symbols()\n",
    "        self.add_input_layer()\n",
    "        self.add_encoder_layer()\n",
    "        self.add_stochastic_layer()\n",
    "        self.add_decoder_hidden()\n",
    "        with tf.variable_scope('decode'):\n",
    "            self.add_decoder_for_training()\n",
    "        with tf.variable_scope('decode', reuse=True):\n",
    "            self.add_decoder_for_inference()\n",
    "        with tf.variable_scope('decode', reuse=True):\n",
    "            self.add_decoder_for_prefix_inference()\n",
    "        self.add_backward_path()\n",
    "    # end method\n",
    "\n",
    "    def add_input_layer(self):\n",
    "        self.X = tf.placeholder(tf.int32, [None, None], name=\"X\")\n",
    "        self.Y = tf.placeholder(tf.int32, [None, None], name=\"Y\")\n",
    "        self.X_seq_len = tf.placeholder(tf.int32, [None], name=\"X_seq_len\")\n",
    "        self.Y_seq_len = tf.placeholder(tf.int32, [None], name=\"Y_seq_len\")\n",
    "        self.input_keep_prob = tf.placeholder(tf.float32,name=\"input_keep_prob\")\n",
    "        self.output_keep_prob = tf.placeholder(tf.float32,name=\"output_keep_prob\")\n",
    "        self.batch_size = tf.shape(self.X)[0]\n",
    "        self.B = tf.placeholder(tf.float32, name='Beta_deterministic_warmup')\n",
    "        self.global_step = tf.Variable(0, name=\"global_step\", trainable=False)\n",
    "    # end method\n",
    "\n",
    "    def single_cell(self, reuse=False):\n",
    "        if self.cell_type == 'lstm':\n",
    "             cell = tf.contrib.rnn.LayerNormBasicLSTMCell(self.rnn_size, reuse=reuse)\n",
    "        else:\n",
    "            cell = tf.contrib.rnn.GRUBlockCell(self.rnn_size)    \n",
    "        cell = tf.contrib.rnn.DropoutWrapper(cell, self.output_keep_prob, self.input_keep_prob)\n",
    "        if self.residual:\n",
    "            cell = myResidualCell.ResidualWrapper(cell)\n",
    "        return cell\n",
    "    \n",
    "    def add_encoder_layer(self):\n",
    "        encoder_embedding = tf.get_variable('encoder_embedding', [len(self.dp.X_w2id), self.encoder_embedding_dim],\n",
    "                                             tf.float32, tf.random_uniform_initializer(-1.0, 1.0))\n",
    "        \n",
    "        self.encoder_inputs = tf.nn.embedding_lookup(encoder_embedding, self.X)\n",
    "        bi_encoder_output, bi_encoder_state = tf.nn.bidirectional_dynamic_rnn(\n",
    "            cell_fw = tf.contrib.rnn.MultiRNNCell([self.single_cell() for _ in range(self.n_layers)]), \n",
    "            cell_bw = tf.contrib.rnn.MultiRNNCell([self.single_cell() for _ in range(self.n_layers)]),\n",
    "            inputs = self.encoder_inputs,\n",
    "            sequence_length = self.X_seq_len,\n",
    "            dtype = tf.float32,\n",
    "            scope = 'bidirectional_rnn')\n",
    "        #print bi_encoder_state\n",
    "        if self.cell_type == 'lstm':\n",
    "            self.encoder_out = tf.concat([bi_encoder_state[0][-1][1],bi_encoder_state[1][-1][1]], -1)\n",
    "        else:\n",
    "            self.encoder_out = tf.concat([bi_encoder_state[0][-1],bi_encoder_state[1][-1]], -1)\n",
    "        \n",
    "    def add_stochastic_layer(self):\n",
    "        # reparametrization trick\n",
    "        self.z_mu = tf.layers.dense(self.encoder_out, self.latent_dim)\n",
    "        z_mean_mu = tf.reduce_mean(self.z_mu, 0)\n",
    "        \n",
    "        z_mean_mu = tf.tile(tf.expand_dims(z_mean_mu,0), [tf.shape(self.z_mu)[0], 1])\n",
    "        \n",
    "        self.aux_loss = tf.nn.relu(float(self.var) - tf.losses.mean_squared_error(z_mean_mu, self.z_mu))\n",
    "        \n",
    "        self.z_lgs2 = tf.layers.dense(self.encoder_out, self.latent_dim)\n",
    "        noise = tf.random_normal(tf.shape(self.z_lgs2))\n",
    "        self.z = self.z_mu + tf.exp(0.5 * self.z_lgs2) * noise\n",
    "        \n",
    "    \n",
    "    def add_decoder_hidden(self):\n",
    "        hidden_state_list = []\n",
    "        for i in range(self.n_layers * 2):\n",
    "            if self.cell_type == 'gru':\n",
    "                hidden_state_list.append(tf.layers.dense(self.z, self.rnn_size))\n",
    "            else:\n",
    "                hidden_state_list.append(tf.contrib.rnn.LSTMStateTuple(tf.layers.dense(self.z, self.rnn_size), tf.layers.dense(self.z, self.rnn_size))) \n",
    "        self.decoder_init_state = tuple(hidden_state_list)\n",
    "        \n",
    "    def processed_decoder_input(self):\n",
    "        main = tf.strided_slice(self.Y, [0, 0], [self.batch_size, -1], [1, 1]) # remove last char\n",
    "        decoder_input = tf.concat([tf.fill([self.batch_size, 1], self._y_go), main], 1)\n",
    "        return decoder_input\n",
    "\n",
    "    def add_decoder_for_training(self):\n",
    "        self.decoder_cell = tf.contrib.rnn.MultiRNNCell([self.single_cell() for _ in range(2 * self.n_layers)])\n",
    "        decoder_embedding = tf.get_variable('decoder_embedding', [len(self.dp.Y_w2id), self.decoder_embedding_dim],\n",
    "                                             tf.float32, tf.random_uniform_initializer(-1.0, 1.0))\n",
    "        emb = tf.nn.embedding_lookup(decoder_embedding, self.processed_decoder_input())\n",
    "        inputs = tf.expand_dims(self.z, 1)\n",
    "        inputs = tf.tile(inputs, [1, tf.shape(emb)[1], 1])\n",
    "        inputs = tf.concat([emb, inputs],2) \n",
    "        training_helper = tf.contrib.seq2seq.TrainingHelper(\n",
    "            inputs = inputs,\n",
    "            sequence_length = self.Y_seq_len,\n",
    "            time_major = False)\n",
    "        training_decoder = tf.contrib.seq2seq.BasicDecoder(\n",
    "            cell = self.decoder_cell,\n",
    "            helper = training_helper,\n",
    "            initial_state = self.decoder_init_state, #self.decoder_cell.zero_state(self.batch_size, tf.float32),\n",
    "            output_layer = core_layers.Dense(len(self.dp.Y_w2id)))\n",
    "        training_decoder_output, training_final_state, _ = tf.contrib.seq2seq.dynamic_decode(\n",
    "            decoder = training_decoder,\n",
    "            impute_finished = True,\n",
    "            maximum_iterations = tf.reduce_max(self.Y_seq_len))\n",
    "        self.training_logits = training_decoder_output.rnn_output\n",
    "        self.init_prefix_state = training_final_state\n",
    "\n",
    "    def add_decoder_for_inference(self):   \n",
    "        decoder_embedding = tf.get_variable('decoder_embedding')\n",
    "        self.beam_f = (lambda ids: tf.concat([tf.nn.embedding_lookup(decoder_embedding, ids), \n",
    "                                    tf.tile(tf.expand_dims(self.z, 1), \n",
    "                                            [1,int(tf.nn.embedding_lookup(decoder_embedding, ids).get_shape()[1]), 1]) if len(ids.get_shape()) !=1 \n",
    "                                             else self.z], -1))\n",
    "\n",
    "        predicting_decoder = tf.contrib.seq2seq.BeamSearchDecoder(\n",
    "            cell = self.decoder_cell,\n",
    "            embedding = self.beam_f, \n",
    "            start_tokens = tf.tile(tf.constant([self._y_go], dtype=tf.int32), [self.batch_size]),\n",
    "            end_token = self._y_eos,\n",
    "            initial_state = tf.contrib.seq2seq.tile_batch(self.decoder_init_state, self.beam_width),#self.decoder_cell.zero_state(self.batch_size * self.beam_width, tf.float32),\n",
    "            beam_width = self.beam_width,\n",
    "            output_layer = core_layers.Dense(len(self.dp.Y_w2id), _reuse=True),\n",
    "            length_penalty_weight = self.beam_penalty)\n",
    "        predicting_decoder_output, _, _ = tf.contrib.seq2seq.dynamic_decode(\n",
    "            decoder = predicting_decoder,\n",
    "            impute_finished = False,\n",
    "            maximum_iterations = self.max_infer_length)\n",
    "        self.predicting_ids = predicting_decoder_output.predicted_ids\n",
    "        self.score = predicting_decoder_output.beam_search_decoder_output.scores\n",
    "        \n",
    "    def add_decoder_for_prefix_inference(self):   \n",
    "        predicting_decoder = tf.contrib.seq2seq.BeamSearchDecoder(\n",
    "            cell = self.decoder_cell,\n",
    "            embedding = self.beam_f,\n",
    "            start_tokens = tf.tile(tf.constant([self._y_go], dtype=tf.int32), [self.batch_size]),\n",
    "            end_token = self._y_eos,\n",
    "            initial_state = tf.contrib.seq2seq.tile_batch(self.init_prefix_state, self.beam_width),\n",
    "            beam_width = self.beam_width,\n",
    "            output_layer = core_layers.Dense(len(self.dp.Y_w2id), _reuse=True),\n",
    "            length_penalty_weight = self.beam_penalty)\n",
    "        \n",
    "        self.prefix_go = tf.placeholder(tf.int32, [None])\n",
    "        prefix_go_beam = tf.tile(tf.expand_dims(self.prefix_go, 1), [1, self.beam_width])\n",
    "        prefix_emb = self.beam_f(prefix_go_beam)\n",
    "        predicting_decoder._start_inputs = prefix_emb\n",
    "        predicting_prefix_decoder_output, _, _ = tf.contrib.seq2seq.dynamic_decode(\n",
    "            decoder = predicting_decoder,\n",
    "            impute_finished = False,\n",
    "            maximum_iterations = self.max_infer_length)\n",
    "        self.predicting_prefix_ids = predicting_prefix_decoder_output.predicted_ids\n",
    "        self.prefix_score = predicting_prefix_decoder_output.beam_search_decoder_output.scores\n",
    "\n",
    "    def add_backward_path(self):\n",
    "        masks = tf.sequence_mask(self.Y_seq_len, tf.reduce_max(self.Y_seq_len), dtype=tf.float32)\n",
    "        self.reconstruct_loss = tf.contrib.seq2seq.sequence_loss(logits = self.training_logits,\n",
    "                                                     targets = self.Y,\n",
    "                                                     weights = masks)\n",
    "        self.all_reconstruct_loss = tf.reduce_sum(tf.contrib.seq2seq.sequence_loss(logits = self.training_logits,\n",
    "                                                     targets = self.Y,\n",
    "                                                     weights = masks,\n",
    "                                                     average_across_timesteps=False))\n",
    "        self.kl_loss = tf.reduce_mean(-0.5 * tf.reduce_sum(1 + self.z_lgs2 - tf.square(self.z_mu) - tf.exp(self.z_lgs2), 1))\n",
    "        self.loss = self.reconstruct_loss + self.B * self.latent_weight * self.kl_loss + self.aux_loss\n",
    "        params = tf.trainable_variables()\n",
    "        gradients = tf.gradients(self.loss, params)\n",
    "        clipped_gradients, _ = tf.clip_by_global_norm(gradients, self.grad_clip)\n",
    "        self.learning_rate = tf.constant(self.lr)\n",
    "        self.learning_rate = self.get_learning_rate_decay(self.decay_scheme)  # decay\n",
    "        self.train_op = tf.train.AdamOptimizer(self.learning_rate).apply_gradients(zip(clipped_gradients, params), global_step=self.global_step)\n",
    "\n",
    "    def register_symbols(self):\n",
    "        self._x_go = self.dp.X_w2id['<GO>']\n",
    "        self._x_eos = self.dp.X_w2id['<EOS>']\n",
    "        self._x_pad = self.dp.X_w2id['<PAD>']\n",
    "        self._x_unk = self.dp.X_w2id['<UNK>']\n",
    "        \n",
    "        self._y_go = self.dp.Y_w2id['<GO>']\n",
    "        self._y_eos = self.dp.Y_w2id['<EOS>']\n",
    "        self._y_pad = self.dp.Y_w2id['<PAD>']\n",
    "        self._y_unk = self.dp.Y_w2id['<UNK>']\n",
    "    \n",
    "    def infer(self, input_word):\n",
    "        if self.is_jieba:\n",
    "            input_word = list(jieba.cut(input_word))\n",
    "        if self.reverse:\n",
    "            input_word = input_word[::-1]\n",
    "        input_indices = [self.dp.X_w2id.get(char, self._x_unk) for char in input_word]\n",
    "        out_indices = self.sess.run(self.predicting_ids, {\n",
    "            self.X: [input_indices], self.X_seq_len: [len(input_indices)], self.output_keep_prob:1, self.input_keep_prob:1})\n",
    "        outputs = []\n",
    "        for idx in range(out_indices.shape[-1]):\n",
    "            eos_id = self.dp.Y_w2id['<EOS>']\n",
    "            ot = out_indices[0,:,idx]\n",
    "            if eos_id in ot:\n",
    "                ot = ot.tolist()\n",
    "                ot = ot[:ot.index(eos_id)]\n",
    "            if self.reverse:\n",
    "                ot = ot[::-1]\n",
    "            output_str = ' '.join([self.dp.Y_id2w.get(i, u'&') for i in ot])\n",
    "            outputs.append(output_str)\n",
    "        return outputs\n",
    "    \n",
    "    def prefix_infer(self, input_word, prefix):\n",
    "        if self.is_jieba:\n",
    "            input_word = list(jieba.cut(input_word))\n",
    "            prefix = list(jieba.cut(prefix))\n",
    "        input_indices_X = [self.dp.X_w2id.get(char, self._x_unk) for char in input_word]\n",
    "        input_indices_Y = [self.dp.Y_w2id.get(char, self._y_unk) for char in prefix]\n",
    "        \n",
    "        prefix_go = []\n",
    "        prefix_go.append(input_indices_Y[-1]) \n",
    "        out_indices, scores = self.sess.run([self.predicting_prefix_ids, self.prefix_score], {\n",
    "            self.X: [input_indices_X], self.X_seq_len: [len(input_indices_X)], self.Y:[input_indices_Y], self.Y_seq_len:[len(input_indices_Y)],\n",
    "            self.prefix_go: prefix_go, self.input_keep_prob:1, self.output_keep_prob:1})\n",
    "        \n",
    "        outputs = []\n",
    "        for idx in range(out_indices.shape[-1]):\n",
    "            eos_id = self.dp.Y_w2id['<EOS>']\n",
    "            ot = out_indices[0,:,idx]\n",
    "            if eos_id in ot:\n",
    "                ot = ot.tolist()\n",
    "                ot = ot[:ot.index(eos_id)]\n",
    "                if self.reverse:\n",
    "                    ot = ot[::-1]\n",
    "            if self.reverse:\n",
    "                output_str = ' '.join([self.dp.Y_id2w.get(i, u'&') for i in ot]) + prefix\n",
    "            else:\n",
    "                output_str = prefix + ' '.join([self.dp.Y_id2w.get(i, u'&') for i in ot])\n",
    "            outputs.append(output_str)\n",
    "        return outputs\n",
    "    \n",
    "    def xToz(self, input_word):\n",
    "        if self.is_jieba:\n",
    "            input_word = list(jieba.cut(input_word))\n",
    "        if self.reverse:\n",
    "            input_word = input_word[::-1]\n",
    "        input_indices = [self.dp.X_w2id.get(char, self._x_unk) for char in input_word]\n",
    "        z = self.sess.run(self.z, {self.X: [input_indices], self.X_seq_len: [len(input_indices)], self.output_keep_prob:1, self.input_keep_prob:1})\n",
    "        return z\n",
    "    # end method\n",
    "    \n",
    "    def zTox(self, z):\n",
    "        out_indices = self.sess.run(self.predicting_ids, {self.batch_size:z.shape[0],\n",
    "            self.z:z, self.output_keep_prob:1, self.input_keep_prob:1})\n",
    "        outputs = []\n",
    "        for idx in range(out_indices.shape[-1]):\n",
    "            eos_id = self.dp.Y_w2id['<EOS>']\n",
    "            ot = out_indices[0,:,idx]\n",
    "            if eos_id in ot:\n",
    "                ot = ot.tolist()\n",
    "                ot = ot[:ot.index(eos_id)]\n",
    "            if self.reverse:\n",
    "                ot = ot[::-1]\n",
    "            output_str = ' '.join([self.dp.Y_id2w.get(i, u'&') for i in ot])\n",
    "            outputs.append(output_str)\n",
    "        return outputs\n",
    "    \n",
    "    def generate(self, batch_size = 6):\n",
    "        out_indices = self.sess.run(self.predicting_ids, { self.batch_size:batch_size,\n",
    "            self.z:np.random.randn(batch_size, self.latent_dim), self.output_keep_prob:1, self.input_keep_prob:1})\n",
    "        outputs = []\n",
    "        for idx in range(out_indices.shape[0]):\n",
    "            eos_id = self.dp.Y_w2id['<EOS>']\n",
    "            ot = out_indices[idx,:,0]   # The 0th beam of each batch \n",
    "            if eos_id in ot:\n",
    "                ot = ot.tolist()\n",
    "                ot = ot[:ot.index(eos_id)]\n",
    "            if self.reverse:\n",
    "                ot = ot[::-1]\n",
    "            output_str = ' '.join([self.dp.Y_id2w.get(i, u'&') for i in ot])\n",
    "            outputs.append(output_str)\n",
    "        return outputs\n",
    "    \n",
    "    def restore(self, path):\n",
    "        self.saver.restore(self.sess, path)\n",
    "        print('restore %s success' % path)\n",
    "        \n",
    "    def get_learning_rate_decay(self, decay_scheme='luong234'):\n",
    "        num_train_steps = self.dp.num_steps\n",
    "        if decay_scheme == \"luong10\":\n",
    "            start_decay_step = int(num_train_steps / 2)\n",
    "            remain_steps = num_train_steps - start_decay_step\n",
    "            decay_steps = int(remain_steps / 10)  # decay 10 times\n",
    "            decay_factor = 0.5\n",
    "        else:\n",
    "            start_decay_step = int(num_train_steps * 2 / 3)\n",
    "            remain_steps = num_train_steps - start_decay_step\n",
    "            decay_steps = int(remain_steps / 4)  # decay 4 times\n",
    "            decay_factor = 0.5\n",
    "        return tf.cond(\n",
    "            self.global_step < start_decay_step,\n",
    "            lambda: self.learning_rate,\n",
    "            lambda: tf.train.exponential_decay(\n",
    "                self.learning_rate,\n",
    "                (self.global_step - start_decay_step),\n",
    "                decay_steps, decay_factor, staircase=True),\n",
    "            name=\"learning_rate_decay_cond\")\n",
    "    \n",
    "    def setup_summary(self):\n",
    "        train_loss = tf.Variable(0.)\n",
    "        tf.summary.scalar('Train_loss', train_loss) \n",
    "        train_aux_loss = tf.Variable(0.)\n",
    "        tf.summary.scalar('Train_aux_loss', train_aux_loss) \n",
    "        train_KL_loss = tf.Variable(0.)\n",
    "        tf.summary.scalar('Train_KL_loss', train_KL_loss)\n",
    "        train_r_loss = tf.Variable(0.)\n",
    "        tf.summary.scalar('Train_R_loss', train_r_loss)\n",
    "        test_loss = tf.Variable(0.)\n",
    "        tf.summary.scalar('Test_loss', test_loss) \n",
    "        test_aux_loss = tf.Variable(0.)\n",
    "        tf.summary.scalar('Test_aux_loss', test_aux_loss)\n",
    "        test_KL_loss = tf.Variable(0.)\n",
    "        tf.summary.scalar('Test_KL_loss', test_KL_loss)\n",
    "        test_r_loss = tf.Variable(0.)\n",
    "        tf.summary.scalar('Test_R_loss', test_r_loss)\n",
    "        beta = tf.Variable(0.)\n",
    "        tf.summary.scalar('Beta', beta)\n",
    "        tf.summary.scalar('lr_rate', self.learning_rate)\n",
    "        tf.summary.histogram(\"z_mu\", self.z_mu)\n",
    "        tf.summary.histogram(\"z_ls2\", self.z_lgs2)\n",
    "        tf.summary.histogram(\"z\", self.z)\n",
    "        \n",
    "        summary_vars = [train_loss, train_aux_loss, train_KL_loss, train_r_loss, test_loss, test_aux_loss, test_KL_loss, test_r_loss, beta]\n",
    "        summary_placeholders = [tf.placeholder(tf.float32) for _ in range(len(summary_vars))]\n",
    "        update_ops = [summary_vars[i].assign(summary_placeholders[i]) for i in range(len(summary_vars))]\n",
    "        summary_op = tf.summary.merge_all()\n",
    "        return summary_placeholders, update_ops, summary_op"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-12-28T07:52:31.656783Z",
     "start_time": "2018-12-28T07:52:31.533799Z"
    },
    "code_folding": [],
    "collapsed": true,
    "run_control": {
     "marked": false
    }
   },
   "outputs": [],
   "source": [
    "class VRAE_DP:\n",
    "    def __init__(self, X_indices, Y_indices, X_w2id, Y_w2id, BATCH_SIZE=256, n_epoch=15):\n",
    "        assert len(X_indices) == len(Y_indices)\n",
    "        num_test = int(len(X_indices) * 0.1)\n",
    "        self.n_epoch = n_epoch\n",
    "        self.X_train = np.array(X_indices[num_test:])\n",
    "        self.Y_train = np.array(Y_indices[num_test:])\n",
    "        self.X_test = np.array(X_indices[:num_test])\n",
    "        self.Y_test = np.array(Y_indices[:num_test])\n",
    "        self.num_batch = int(len(self.X_train) / BATCH_SIZE)\n",
    "        self.num_steps = self.num_batch * self.n_epoch\n",
    "        self.batch_size = BATCH_SIZE\n",
    "        self.X_w2id = X_w2id\n",
    "        self.X_id2w = dict(zip(X_w2id.values(), X_w2id.keys()))\n",
    "        self.Y_w2id = Y_w2id\n",
    "        self.Y_id2w = dict(zip(Y_w2id.values(), Y_w2id.keys()))\n",
    "        self._x_pad = self.X_w2id['<PAD>']\n",
    "        self._y_pad = self.Y_w2id['<PAD>']\n",
    "        print('Train_data: %d | Test_data: %d | Batch_size: %d | Num_batch: %d | X_vocab_size: %d | Y_vocab_size: %d' % (len(self.X_train), len(self.X_test), BATCH_SIZE, self.num_batch, len(self.X_w2id), len(self.Y_w2id)))\n",
    "        \n",
    "    def next_batch(self, X, Y):\n",
    "        r = np.random.permutation(len(X))\n",
    "        X = X[r]\n",
    "        Y = Y[r]\n",
    "        for i in range(0, len(X) - len(X) % self.batch_size, self.batch_size):\n",
    "            X_batch = X[i : i + self.batch_size]\n",
    "            Y_batch = Y[i : i + self.batch_size]\n",
    "            padded_X_batch, X_batch_lens = self.pad_sentence_batch(X_batch, self._x_pad)\n",
    "            padded_Y_batch, Y_batch_lens = self.pad_sentence_batch(Y_batch, self._y_pad)\n",
    "            yield (np.array(padded_X_batch),\n",
    "                   np.array(padded_Y_batch),\n",
    "                   X_batch_lens,\n",
    "                   Y_batch_lens)\n",
    "    \n",
    "    def sample_test_batch(self):\n",
    "        padded_X_batch, X_batch_lens = self.pad_sentence_batch(self.X_test[: self.batch_size], self._x_pad)\n",
    "        padded_Y_batch, Y_batch_lens = self.pad_sentence_batch(self.Y_test[: self.batch_size], self._y_pad)\n",
    "        return np.array(padded_X_batch), np.array(padded_Y_batch), X_batch_lens, Y_batch_lens\n",
    "        \n",
    "    def pad_sentence_batch(self, sentence_batch, pad_int):\n",
    "        padded_seqs = []\n",
    "        seq_lens = []\n",
    "        max_sentence_len = max([len(sentence) for sentence in sentence_batch])\n",
    "        for sentence in sentence_batch:\n",
    "            padded_seqs.append(sentence + [pad_int] * (max_sentence_len - len(sentence)))\n",
    "            seq_lens.append(len(sentence))\n",
    "        return padded_seqs, seq_lens\n",
    "    \n",
    "   \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-12-28T07:52:34.027192Z",
     "start_time": "2018-12-28T07:52:31.658855Z"
    },
    "code_folding": [],
    "collapsed": true,
    "run_control": {
     "marked": true
    }
   },
   "outputs": [],
   "source": [
    "import scipy.interpolate as si\n",
    "from scipy import interpolate\n",
    "\n",
    "\n",
    "def BetaGenerator(epoches, beta_decay_period, beta_decay_offset):\n",
    "    points = [[0,0], [0, beta_decay_offset],[0, beta_decay_offset + 0.33 * beta_decay_period], [1, beta_decay_offset + 0.66*beta_decay_period],[1, beta_decay_offset + beta_decay_period], [1, epoches] ];\n",
    "    points = np.array(points)\n",
    "    x = points[:,0]\n",
    "    y = points[:,1]\n",
    "    t = range(len(points))\n",
    "    ipl_t = np.linspace(0.0, len(points) - 1, 100)\n",
    "    x_tup = si.splrep(t, x, k=3)\n",
    "    y_tup = si.splrep(t, y, k=3)\n",
    "    x_list = list(x_tup)\n",
    "    xl = x.tolist()\n",
    "    x_list[1] = xl + [0.0, 0.0, 0.0, 0.0]\n",
    "    y_list = list(y_tup)\n",
    "    yl = y.tolist()\n",
    "    y_list[1] = yl + [0.0, 0.0, 0.0, 0.0]\n",
    "    x_i = si.splev(ipl_t, x_list)\n",
    "    y_i = si.splev(ipl_t, y_list)\n",
    "    return interpolate.interp1d(y_i, x_i)\n",
    "\n",
    "class VRAE_util:\n",
    "    def __init__(self, dp, model, display_freq=3):\n",
    "        self.display_freq = display_freq\n",
    "        self.dp = dp\n",
    "        self.model = model\n",
    "        self.summary_cnt = 0\n",
    "        self.betaG = BetaGenerator(self.dp.n_epoch*self.dp.num_batch, self.model.beta_decay_period*self.dp.num_batch, self.model.beta_decay_offset*self.dp.num_batch)\n",
    "        \n",
    "    def train(self, epoch):\n",
    "        avg_loss = 0.0\n",
    "        avg_all_loss = 0.0\n",
    "        avg_r_loss = 0.0\n",
    "        avg_kl_loss = 0.0\n",
    "        avg_aux_loss = 0.0\n",
    "        tic = time.time()\n",
    "        X_test_batch, Y_test_batch, X_test_batch_lens, Y_test_batch_lens = self.dp.sample_test_batch()\n",
    "        for local_step, (X_train_batch, Y_train_batch, X_train_batch_lens, Y_train_batch_lens) in enumerate(\n",
    "            self.dp.next_batch(self.dp.X_train, self.dp.Y_train)):\n",
    "            beta = 0.001 + self.betaG(self.model.step) # add small value to avoid points to scatter\n",
    "            self.model.step, _, loss, all_loss, r_loss, kl_loss, aux_loss = self.model.sess.run([self.model.global_step, self.model.train_op, \n",
    "                                                            self.model.loss, self.model.all_reconstruct_loss, self.model.reconstruct_loss, self.model.kl_loss, self.model.aux_loss], \n",
    "                                          {self.model.X: X_train_batch,\n",
    "                                           self.model.Y: Y_train_batch,\n",
    "                                           self.model.X_seq_len: X_train_batch_lens,\n",
    "                                           self.model.Y_seq_len: Y_train_batch_lens,\n",
    "                                           self.model.output_keep_prob:self.model._output_keep_prob,\n",
    "                                           self.model.input_keep_prob:self.model._input_keep_prob,\n",
    "                                          self.model.B:beta})\n",
    "            avg_loss += loss\n",
    "            avg_all_loss += all_loss\n",
    "            avg_r_loss += r_loss\n",
    "            avg_kl_loss += kl_loss\n",
    "            avg_aux_loss += aux_loss\n",
    "            # summary\n",
    "            if local_step % 10 == 0:\n",
    "                self.summary_cnt += 1\n",
    "                val_loss, val_r_loss, val_kl_loss, val_aux_loss = self.model.sess.run([self.model.loss, self.model.reconstruct_loss, self.model.kl_loss, self.model.aux_loss], \n",
    "                                               {self.model.X: X_test_batch,\n",
    "                                                     self.model.Y: Y_test_batch,\n",
    "                                                     self.model.X_seq_len: X_test_batch_lens,\n",
    "                                                     self.model.Y_seq_len: Y_test_batch_lens,\n",
    "                                                     self.model.output_keep_prob:1,\n",
    "                                                     self.model.input_keep_prob:1,\n",
    "                                                     self.model.B:beta})\n",
    "                stats = [avg_loss/(local_step+1), avg_aux_loss/(local_step+1), avg_kl_loss/(local_step+1), avg_r_loss/(local_step+1),\n",
    "                         val_loss, val_aux_loss, val_kl_loss, val_r_loss, beta]\n",
    "                for i in range(len(stats)):\n",
    "                    self.model.sess.run(self.model.update_ops[i], feed_dict={\n",
    "                        self.model.summary_placeholders[i]: float(stats[i])\n",
    "                    })\n",
    "                summary_str = self.model.sess.run(self.model.summary_op, {\n",
    "                    self.model.X: X_test_batch, \n",
    "                    self.model.X_seq_len: X_test_batch_lens,\n",
    "                    self.model.output_keep_prob:1,\n",
    "                    self.model.input_keep_prob:1})\n",
    "                self.summary_writer.add_summary(summary_str, self.summary_cnt)\n",
    "                \n",
    "            if local_step % (self.dp.num_batch / self.display_freq) == 0:\n",
    "                val_loss, val_r_loss, val_kl_loss, val_aux_loss = self.model.sess.run([self.model.loss, self.model.reconstruct_loss, self.model.kl_loss, self.model.aux_loss], \n",
    "                                               {self.model.X: X_test_batch,\n",
    "                                                     self.model.Y: Y_test_batch,\n",
    "                                                     self.model.X_seq_len: X_test_batch_lens,\n",
    "                                                     self.model.Y_seq_len: Y_test_batch_lens,\n",
    "                                                     self.model.output_keep_prob:1,\n",
    "                                                     self.model.input_keep_prob:1,\n",
    "                                                     self.model.B:beta})\n",
    "                print(\"Epoch %d/%d | Batch %d/%d | Train_loss: %.3f = %.3f(%.3f) + %.3f +%.3f | Test_loss: %.3f = %.3f + %.3f + %.3f | Time_cost:%.3f\" % (epoch, self.n_epoch, local_step, self.dp.num_batch, \n",
    "                                                                                                                                       avg_loss / (local_step + 1),\n",
    "                                                                                                                                       avg_r_loss / (local_step + 1),\n",
    "                                                                                                                                       avg_all_loss / (local_step + 1),\n",
    "                                                                                                                                       avg_kl_loss / (local_step + 1),\n",
    "                                                                                                                                       avg_aux_loss / (local_step + 1),\n",
    "                                                                                                                                       val_loss, val_r_loss, val_kl_loss, val_aux_loss, time.time()-tic))\n",
    "                self.cal()\n",
    "                tic = time.time()\n",
    "        return avg_loss / self.dp.num_batch, avg_all_loss / self.dp.num_batch, avg_r_loss / self.dp.num_batch, avg_kl_loss / self.dp.num_batch, avg_aux_loss / self.dp.num_batch\n",
    "    \n",
    "    def test(self):\n",
    "        avg_loss = 0.0\n",
    "        avg_r_loss = 0.0\n",
    "        avg_all_loss = 0.0\n",
    "        avg_kl_loss = 0.0\n",
    "        avg_aux_loss = 0.0\n",
    "        beta = 0.001 + self.betaG(self.model.step) # add small value to avoid points to scatter\n",
    "        for local_step, (X_test_batch, Y_test_batch, X_test_batch_lens, Y_test_batch_lens) in enumerate(\n",
    "            self.dp.next_batch(self.dp.X_test, self.dp.Y_test)):\n",
    "            val_loss, val_all_loss, val_r_loss, val_kl_loss, val_aux_loss = self.model.sess.run([self.model.loss, self.model.all_reconstruct_loss, self.model.reconstruct_loss, self.model.kl_loss, self.model.aux_loss], \n",
    "                                               {self.model.X: X_test_batch,\n",
    "                                                     self.model.Y: Y_test_batch,\n",
    "                                                     self.model.X_seq_len: X_test_batch_lens,\n",
    "                                                     self.model.Y_seq_len: Y_test_batch_lens,\n",
    "                                                     self.model.output_keep_prob:1,\n",
    "                                                     self.model.input_keep_prob:1,\n",
    "                                                     self.model.B:beta})\n",
    "            avg_loss += val_loss\n",
    "            avg_all_loss += val_all_loss\n",
    "            avg_r_loss += val_r_loss\n",
    "            avg_kl_loss += val_kl_loss\n",
    "            avg_aux_loss += val_aux_loss\n",
    "        return avg_loss / (local_step + 1), avg_all_loss / (local_step + 1), avg_r_loss / (local_step + 1), avg_kl_loss / (local_step + 1) , avg_aux_loss / (local_step + 1)\n",
    "    \n",
    "    def fit(self, train_dir, is_bleu=False):\n",
    "        self.n_epoch = self.dp.n_epoch\n",
    "        test_loss_list = []\n",
    "        train_loss_list = []\n",
    "        test_r_loss_list = []\n",
    "        test_aux_loss_list = []\n",
    "        train_r_loss_list = []\n",
    "        test_kl_loss_list = []\n",
    "        train_kl_loss_list = []\n",
    "        train_aux_loss_list = []\n",
    "        time_cost_list = []\n",
    "        bleu_list = []\n",
    "        timestamp = str(int(time.time()))\n",
    "        out_dir = train_dir#os.path.abspath(os.path.join(train_dir, \"runs\", 'var_2_1'))\n",
    "        if not os.path.exists(out_dir):\n",
    "            os.makedirs(out_dir)\n",
    "        print(\"Writing to %s\" % out_dir)\n",
    "        checkpoint_prefix = os.path.join(out_dir, \"model\")\n",
    "        self.summary_writer = tf.summary.FileWriter(os.path.join(out_dir, 'Summary'), self.model.sess.graph)\n",
    "        for epoch in range(1, self.n_epoch+1):\n",
    "            tic = time.time()\n",
    "            train_loss, train_all_loss, train_r_loss, train_kl_loss, train_aux_loss = self.train(epoch)\n",
    "            train_loss_list.append(train_loss)\n",
    "            train_r_loss_list.append(train_r_loss)\n",
    "            train_kl_loss_list.append(train_kl_loss)\n",
    "            train_aux_loss_list.append(train_aux_loss)\n",
    "            \n",
    "            test_loss, test_all_loss, test_r_loss, test_kl_loss, test_aux_loss = self.test()\n",
    "            test_loss_list.append(test_loss)\n",
    "            test_r_loss_list.append(test_r_loss)\n",
    "            test_kl_loss_list.append(test_kl_loss)\n",
    "            test_aux_loss_list.append(test_aux_loss)\n",
    "            toc = time.time()\n",
    "            time_cost_list.append((toc - tic))\n",
    "            if is_bleu:\n",
    "                bleu = self.test_bleu()\n",
    "                bleu_list.append(bleu)\n",
    "                print(\"Epoch %d/%d | Train_loss: %.3f = %.3f(%.3f) + %.3f + %.3f | Test_loss: %.3f = %.3f(%.3f) + %.3f + %.3f | Bleu: %.3f\" % (epoch, self.n_epoch, train_loss, train_r_loss, train_all_loss, train_kl_loss, train_aux_loss, test_loss, test_r_loss, test_all_loss, test_kl_loss, test_aux_loss, bleu))\n",
    "            else:\n",
    "                bleu = 0.0\n",
    "                print(\"Epoch %d/%d | Train_loss: %.3f = %.3f(%.3f) + %.3f + %.3f | Test_loss: %.3f = %.3f(%.3f) + %.3f + %.3f | Bleu: %.3f\" % (epoch, self.n_epoch, train_loss, train_r_loss, train_all_loss, train_kl_loss, train_aux_loss, test_loss, test_r_loss, test_all_loss, test_kl_loss, test_aux_loss, bleu))\n",
    "            if self.model.is_save:\n",
    "                cPickle.dump((train_loss_list, train_r_loss_list, train_kl_loss_list, train_aux_loss_list, test_loss_list, test_r_loss_list, test_kl_loss_list, test_aux_loss,time_cost_list, bleu_list), open(os.path.join(out_dir,\"res.pkl\"),'wb'))\n",
    "                path = self.model.saver.save(self.model.sess, checkpoint_prefix, global_step=epoch)\n",
    "                print(\"Saved model checkpoint to %s\" % path)\n",
    "    \n",
    "    def show(self, sent, id2w):\n",
    "        return \" \".join([id2w.get(idx, u'&') for idx in sent])\n",
    "    \n",
    "    def cal(self, n_example=5):\n",
    "        train_n_example = int(n_example / 2)\n",
    "        test_n_example = n_example - train_n_example\n",
    "        for _ in range(test_n_example):\n",
    "            example = self.show(self.dp.X_test[_], self.dp.X_id2w)\n",
    "            y = self.show(self.dp.Y_test[_], self.dp.Y_id2w)\n",
    "            o = self.model.infer(example)[0]\n",
    "            print('Test_Input: %s | Output: %s | GroundTruth: %s' % (example, o, y))\n",
    "        for _ in range(train_n_example):\n",
    "            example = self.show(self.dp.X_train[_], self.dp.X_id2w)\n",
    "            y = self.show(self.dp.Y_train[_], self.dp.Y_id2w)\n",
    "            o = self.model.infer(example)[0]\n",
    "            print('Train_Input: %s | Output: %s | GroundTruth: %s' % (example, o, y)) \n",
    "        o = self.model.generate()\n",
    "        print('generate:')\n",
    "        for oo in o:\n",
    "            print('【',oo,'】')\n",
    "        print(\"\")\n",
    "        \n",
    "    def test_bleu(self, N=300, gram=4):\n",
    "        all_score = []\n",
    "        for i in range(N):\n",
    "            input_indices = self.show(self.dp.X_test[i], self.dp.X_id2w)\n",
    "            o = self.model.infer(input_indices)[0]\n",
    "            refer4bleu = [[' '.join([self.dp.Y_id2w.get(w, u'&') for w in self.dp.Y_test[i]])]]\n",
    "            candi = [' '.join(w for w in o)]\n",
    "            score = BLEU(candi, refer4bleu, gram=gram)\n",
    "            all_score.append(score)\n",
    "        return np.mean(all_score)\n",
    "    \n",
    "    def show_res(self, path):\n",
    "        res = cPickle.load(open(path))\n",
    "        plt.figure(1)\n",
    "        plt.title('The train results') \n",
    "        l1, = plt.plot(res[0], 'g')\n",
    "        l2, = plt.plot(res[1], 'r')\n",
    "        l3, = plt.plot(res[2], 'b')\n",
    "        plt.legend(handles = [l1, l2, l3], labels = [\"Train_loss\",\"Train_r_loss\",\"Train_kl_loss\"], loc = 'best')\n",
    "        plt.show()\n",
    "        \n",
    "        plt.figure(1)\n",
    "        plt.title('The test results') \n",
    "        l4, = plt.plot(res[3], 'g')\n",
    "        l5, = plt.plot(res[4], 'r')\n",
    "        l6, = plt.plot(res[5], 'r')\n",
    "        l7, = plt.plot(res[-1], 'b')\n",
    "        plt.legend(handles = [l4, l5, l6, l7], labels = [\"Test_loss\",\"Test_r_loss\",\"Test_kl_loss\",\"BLEU\"], loc = 'best')\n",
    "        plt.show()\n",
    "        \n",
    "    def test_all(self, path, epoch_range, is_bleu=True):\n",
    "        val_loss_list = []\n",
    "        bleu_list = []\n",
    "        for i in range(epoch_range[0], epoch_range[-1]):\n",
    "            self.model.restore(path + str(i))\n",
    "            val_loss = self.test()\n",
    "            val_loss_list.append(val_loss)\n",
    "            if is_bleu:\n",
    "                bleu_score = self.test_bleu()\n",
    "                bleu_list.append(bleu_score)\n",
    "        plt.figure(1)\n",
    "        plt.title('The results') \n",
    "        l1, = plt.plot(val_loss_list,'r')\n",
    "        l2, = plt.plot(bleu_list,'b')\n",
    "        plt.legend(handles = [l1, l2], labels = [\"Test_loss\",\"BLEU\"], loc = 'best')\n",
    "        plt.show()\n",
    "        \n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-12-28T07:52:37.348805Z",
     "start_time": "2018-12-28T07:52:34.029218Z"
    },
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "w2id, id2w = cPickle.load(open(\"Data/SQUAD/w2id_id2w.pkl\",'rb'))\n",
    "data = cPickle.load(open(\"Data/SQUAD/X_indices.pkl\",'rb'))\n",
    "data_X = [o[:-1] for o in data]\n",
    "\n",
    "for t in random.sample(data, 3):\n",
    "    print(' '.join(id2w[idx] for idx in t))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-12-28T07:54:18.947474Z",
     "start_time": "2018-12-28T07:52:37.350896Z"
    },
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "\n",
    "dp = VRAE_DP(data_X, data, w2id, w2id)\n",
    "g = tf.Graph() \n",
    "sess = tf.Session(graph=g) \n",
    "with sess.as_default():\n",
    "    with sess.graph.as_default():\n",
    "        model = VRAE(\n",
    "            dp = dp,\n",
    "            rnn_size = 1024,\n",
    "            latent_dim = 16,\n",
    "            n_layers = 1,\n",
    "            var = 2.5,\n",
    "            encoder_embedding_dim = 512,\n",
    "            decoder_embedding_dim = 512,\n",
    "            cell_type='gru',\n",
    "            max_infer_length=35,\n",
    "            residual=False,\n",
    "            is_jieba=False,\n",
    "            is_save=True,\n",
    "            beam_width=1,\n",
    "            sess= sess\n",
    "        )\n",
    "        util = VRAE_util(dp=dp, model=model)\n",
    "        util.fit(train_dir='Models/')  #Training\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "model.generate()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "model.restore('Models/model-10')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "#model.load()\n",
    "for i, x in enumerate(data_X):\n",
    "    x_str = ' '.join([id2w[ids] for ids in x])\n",
    "    print(x_str)\n",
    "    print(model.infer(x_str.split()))\n",
    "    print('')\n",
    "    if i == 50:\n",
    "        break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#model.load()\n",
    "model.restore('Models/model-10')\n",
    "data = cPickle.load(open(\"Data/SQUAD/X_indices.pkl\",'rb'))\n",
    "data_X = [o[:-1] for o in data]\n",
    "with open('vae_10.txt', 'w', encoding='utf-8') as fout:\n",
    "    for i, x in enumerate(data_X):\n",
    "        x_str = ' '.join([id2w[ids] for ids in x])\n",
    "        #print(x_str)\n",
    "        o_str = model.infer(x_str.split())[0]\n",
    "        fout.write('{}\\n'.format(o_str))\n",
    "        #print('')\n",
    "        if i % 100 == 0:\n",
    "            print(i)\n",
    "        #if i == 50:\n",
    "        #    break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "hide_input": false,
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.10"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {
    "height": "calc(100% - 180px)",
    "left": "10px",
    "top": "150px",
    "width": "165px"
   },
   "toc_section_display": true,
   "toc_window_display": true
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
